package cn.le.crawler.ipproxy;

import cn.le.crawler.AbstractCrawler;
import okhttp3.Request;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import java.util.LinkedList;
import java.util.List;

@Component
public class IpKuaidailiCrawler extends AbstractCrawler {

    private static IpKuaidailiCrawler crawler;

    private static final String URL = " https://www.kuaidaili.com/free/inha/";

    public static List<IpProxyData> crawler() throws Exception {
        List<IpProxyData> list = new LinkedList<>();
        for (int i = 1; i < 200; i++) {
            System.out.println("proxy --->" + i);
            String utl = URL + i;
            String context = getResult(getRequest(utl));
            Document document = parseDocument(context);
            list.addAll(parsingData(document));
            Thread.sleep(1000);
        }
        return list;
    }

    private static List<IpProxyData> parsingData(Document document){
        List<IpProxyData> list = new LinkedList<>();
        Elements elements = document.getElementById("list").select("table").select("tbody").select("tr");
        elements.forEach(element -> {
            Elements td = element.select("td");
            String ip = td.get(0).text();
            String port = td.get(1).text();
            String type = td.get(3).text();

            IpProxyData proxy = new IpProxyData();
            proxy.setIp(ip);
            proxy.setPort(Integer.valueOf(port));
            proxy.setType(type);
            list.add(proxy);
        });
        return list;
    }


    public static Request getRequest(String url){
        Request request = new Request.Builder().url(url)
                .addHeader("accept", "*/*")
                .addHeader("connection", "Keep-Alive")
                .addHeader("contentType", "utf-8")
                .addHeader("accept-language","zh-CN,zh;q=0.9,en;q=0.8")
                .addHeader("cookie","__jdu=15809783057681031157505; shshshfpa=0884a917-087f-2467-f57b-23d24960558b-1581483701; shshshfpb=h2y2xZ8YKZhvVgPUZlI9UYQ%3D%3D; 3AB9D23F7A4B3C9B=25PZKOJ2CJDZMAYJ2GMHVSPVWITOLXBAP3QA6CLK2XRNW6BAGIU6THOAVWMYGUSLXXTX7GVDS2ABMZAU7LV6RAU3GM; mba_muid=15809783057681031157505; shshshfp=efbbce1938df46b4ed4206a9ddf8e288; __jdv=122270672|baidu|-|organic|not set|1582374387540; areaId=22; ipLoc-djd=22-1930-49327-0; RT=\"z=1&dm=jd.com&si=yfd0ujpazmp&ss=k74vj904&sl=0&tt=0&r=0bfa57d2e173fc3acc2eb927facc6619&ul=r9038&hd=r9045\"; __jda=122270672.15809783057681031157505.1580978306.1582868795.1582972039.28; __jdc=122270672; __jdb=122270672.2.15809783057681031157505|28.1582972039")
                .addHeader("referer","https://auction.jd.com/sifa_list.html?childrenCateId=12728")
                .addHeader("sec-fetch-dest","script")
                .addHeader("sec-fetch-mode","no-cors")
                .addHeader("sec-fetch-site","same-site")
                .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36")
                .get()
                .build();
        return request;
    }


    @PostConstruct
    private void init(){
        crawler = this;
    }

}
